# This is a part of an crawler implementatoins
#
# Author::    Marco Gemeli  (gemelli@cs.unibo.it)


require 'rubygems'
require 'curb'

module Crawler
  
  # This is an abstrac crawler. It's has a base structure to parse a stream of services info
  # and the crawl from the web the necessary informations
  class Crawler
    
    public
    
    # instance the crawler, the options avaleable are:
    # * <b>:conditions</b> the SQL conditions that must be verify to retrive a cawl from the web. <i>Default "status is null or status <> 200"</i> 
    # * <b>:stop_crawling?</b> is a proedure lambda {|crwal| return <i>[false|true]</i>} <i>Default lambda {|c| false}</i>
    def initialize(options = {})
      @curl = Curl::Easy.new
      
      if options[:conditions].nil? then
        @conditions = "status is null or status <> 200"
      else
        @conditions = options[:conditions]
      end
      
      if options[:stop_crawling?].nil? then
        @stop_crawling = lambda {|c| false}
      else
        @stop_crawling = options[:stop_crawling?]
      end
      
      if options[:min_delay].nil? then
        @min_delay = 0
      else
        if options[:min_delay].integer? && options[:min_delay].to_i >= 0 then
          @min_delay = options[:min_delay].to_i
        else
          raise ArgumentError.new ":min_delay must be an integere >= 0, you gaved #{options[:min_delay]}"
        end
      end
      
      if options[:max_delay].nil? then
        @max_delay = 0
      else
        if options[:max_delay].integer? && options[:max_delay].to_i >= 0 then
          @max_delay = options[:max_delay].to_i
        else
          raise ArgumentError.new "max_delay must be an integere >= 0, you gaved #{options[:max_delay]}"
        end
      end
      
      raise ArgumentError.new "max_delay must be grether or equalls to min_delay, you gaved max_delay=#{@max_delay} min_delay=#{@min_delay}" if @max_delay < @min_delay
    end
    
    def start_crawling(&block)
      log = {:crwaled_success => 0, :crwaled_faild => 0, :stopped => false}
      crawl_table.find_each(:conditions => conditions) { |c|
        err = false
        
        begin
          response = crawl(c)
        rescue Exception => e
          c.status = 1
          c.body = "#{e.to_s}\n#{e.backtrace.join("\n")}"
          err = true
          response = false
        end
        
        c.crawled_at = DateTime.now.strftime "%F %T"
        c.save
        
        if response then
          log[:crwaled_success] += 1
        else
          log[:crwaled_faild] += 1
        end
        
        yield c if block
        if err then
          $stderr.puts "#error in #<#{c.class} id: #{c.id}>"
          $stderr.puts c.body
        end
        
        if stop_crawling? c then
          log[:stopped] = true
          break
        end
        wait
      }
      log
    end
    
    def reset_http_session
      @curl = Curl::Easy.new
    end
    
    def update_urls
      crawled = 0
      Service.find_each(:conditions => update_url_conditios) { |s|
        if not has_crawl? s then
          crawl_table.create :service_id => s.id
          crawled +=1
        end
      }
      crawled
    end   
    
    def conditions
      @conditions
    end
    
    def update_url_conditios 
      ['ff_id = ?', service_name]
    end
    
    def service_name
      raise NotImplementedError.new "Subclass must implents this method!!"
    end
    
    def crawl_table
      raise NotImplementedError.new "Subclass must implents this method!!"
    end
    
    protected
    
    def crawl(c)
      raise NotImplementedError.new "Subclass must implents this method!!"
    end
    
    def has_crawl?(s)
      false
    end
    
    def stop_crawling?(r)
      @stop_crawling.call r
    end  
    
    def wait
      delay = rand @max_delay - @min_delay + 1 + @min_delay
      sleep delay
    end
    
  end
  
end